





<!DOCTYPE html>
<html class="writer-html5" lang="zh-CN" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>microTVM设计文档 &mdash; tvm 0.8.dev1982 文档</title>
  

  
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tlcpack_theme.css" type="text/css" />

  
  
    <link rel="shortcut icon" href="../_static/tvm-logo-square.png"/>
  

  
  
  
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/translations.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <script type="text/javascript" src="../_static/js/tlcpack_theme.js"></script>
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="Model Library Format" href="model_library_format.html" />
    <link rel="prev" title="Security Guide" href="security.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    
<header class="header">
    <div class="innercontainer">
      <div class="headerInner d-flex justify-content-between align-items-center">
          <div class="headerLogo">
               <a href="https://tvm.apache.org/"><img src=https://tvm.apache.org/assets/images/logo.svg alt="logo"></a>
          </div>

          <div id="headMenu" class="headerNav">
            <button type="button" id="closeHeadMenu" class="navCloseBtn"><img src="../_static/img/close-icon.svg" alt="Close"></button>
             <ul class="nav">
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/community>Community</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/download>Download</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/vta>VTA</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/blog>Blog</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/docs>Docs</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmconf.org>Conference</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://github.com/apache/tvm/>Github</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmchinese.github.io/declaration_zh_CN.html>About-Translators</a>
                </li>
             </ul>
               <div class="responsivetlcdropdown">
                 <button type="button" class="btn-link">
                   ASF
                 </button>
                 <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                 </ul>
               </div>
          </div>
            <div class="responsiveMenuIcon">
              <button type="button" id="menuBtn" class="btn-menu"><img src="../_static/img/menu-icon.svg" alt="Menu Icon"></button>
            </div>

            <div class="tlcDropdown">
              <div class="dropdown">
                <button type="button" class="btn-link dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                  ASF
                </button>
                <div class="dropdown-menu dropdown-menu-right">
                  <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                  </ul>
                </div>
              </div>
          </div>
       </div>
    </div>
 </header>
 
    <nav data-toggle="wy-nav-shift" class="wy-nav-side fixed">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html">
          

          
            
            <img src="../_static/tvm-logo-small.png" class="logo" alt="Logo"/>
          
          </a>

          
            
            
                <div class="version">
                  0.8.dev1982
                </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption" role="heading"><span class="caption-text">如何开始</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../install/index.html">安装 TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contribute/index.html">贡献者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">用户引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorial/index.html">User Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../how_to/index.html">How To Guides</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">开发者引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../dev/tutorial/index.html">Developer Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dev/how_to/how_to.html">开发者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">架构指南</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="index.html">Design and Architecture</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="index.html#example-compilation-flow">编译流程示例</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#logical-architecture-components">逻辑架构组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-support">tvm/support</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-runtime">tvm/runtime</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-node">tvm/node</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-ir">tvm/ir</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-target">tvm/target</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-tir">tvm/tir</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-arith">tvm/arith</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-te">tvm/te</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-topi">tvm/topi</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-relay">tvm/relay</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-autotvm">tvm/autotvm</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#frontends">Frontends</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#security">安全</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html#microtvm">microTVM</a><ul class="current">
<li class="toctree-l3 current"><a class="current reference internal" href="#">microTVM设计文档</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#background">背景知识</a></li>
<li class="toctree-l4"><a class="reference internal" href="#typical-use">典型案例</a></li>
<li class="toctree-l4"><a class="reference internal" href="#design-goals">Design Goals</a></li>
<li class="toctree-l4"><a class="reference internal" href="#overview">Overview</a></li>
<li class="toctree-l4"><a class="reference internal" href="#future-work">Future Work</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="model_library_format.html">Model Library Format</a></li>
</ul>
</li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">主题引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../topic/microtvm/index.html">microTVM：裸机使用TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../topic/vta/index.html">VTA: Versatile Tensor Accelerator</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">参考指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../reference/langref/index.html">语言参考</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/python/index.html">Python API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/links.html">Other APIs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/publications.html">Publications</a></li>
<li class="toctree-l1"><a class="reference internal" href="../genindex.html">索引</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
      
      <nav class="wy-nav-top" aria-label="top navigation" data-toggle="wy-nav-top">
        
            <div class="togglemenu">

            </div>
            <div class="nav-content">
              <!-- tvm -->
              Table of content
            </div>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        

          




















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> <span class="br-arrow">></span></li>
        
          <li><a href="index.html">Design and Architecture</a> <span class="br-arrow">></span></li>
        
      <li>microTVM设计文档</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/arch/microtvm_design.rst.txt" rel="nofollow"> <img src="../_static//img/source.svg" alt="viewsource"/></a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="microtvm-design-document">
<span id="microtvm-design"></span><h1><a class="toc-backref" href="#id1">microTVM设计文档</a><a class="headerlink" href="#microtvm-design-document" title="永久链接至标题">¶</a></h1>
<div class="contents topic" id="table-of-contents">
<p class="topic-title">目录</p>
<ul class="simple">
<li><p><a class="reference internal" href="#microtvm-design-document" id="id1">microTVM设计文档</a></p>
<ul>
<li><p><a class="reference internal" href="#background" id="id2">背景知识</a></p></li>
<li><p><a class="reference internal" href="#typical-use" id="id3">典型案例</a></p></li>
<li><p><a class="reference internal" href="#design-goals" id="id4">Design Goals</a></p></li>
<li><p><a class="reference internal" href="#overview" id="id5">Overview</a></p>
<ul>
<li><p><a class="reference internal" href="#modeling-target-platforms" id="id6">Modeling Target Platforms</a></p></li>
<li><p><a class="reference internal" href="#tvm-targets-for-microtvm" id="id7">TVM Targets for microTVM</a></p></li>
<li><p><a class="reference internal" href="#writing-schedules-for-microtvm" id="id8">Writing Schedules for microTVM</a></p></li>
<li><p><a class="reference internal" href="#executing-models" id="id9">Executing Models</a></p></li>
<li><p><a class="reference internal" href="#microtvm-firmware" id="id10">microTVM Firmware</a></p></li>
<li><p><a class="reference internal" href="#parts-of-a-microtvm-binary" id="id11">Parts of a microTVM Binary</a></p></li>
<li><p><a class="reference internal" href="#the-automated-build-flow" id="id12">The Automated Build Flow</a></p></li>
<li><p><a class="reference internal" href="#measuring-operator-performance" id="id13">Measuring operator performance</a></p></li>
</ul>
</li>
<li><p><a class="reference internal" href="#future-work" id="id14">Future Work</a></p>
<ul>
<li><p><a class="reference internal" href="#ahead-of-time-runtime" id="id15">Ahead-of-Time Runtime</a></p></li>
<li><p><a class="reference internal" href="#memory-planning" id="id16">Memory Planning</a></p></li>
<li><p><a class="reference internal" href="#heterogeneous-execution" id="id17">Heterogeneous Execution</a></p></li>
<li><p><a class="reference internal" href="#autotuning-target" id="id18">Autotuning Target</a></p></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="background">
<h2><a class="toc-backref" href="#id2">背景知识</a><a class="headerlink" href="#background" title="永久链接至标题">¶</a></h2>
<p>TVM 是一个模型部署框架，在传统操作系统的各种模型中都表现出了良好的性能。鉴于 TVM 的分层编译方法，将TVM应用到微控制器设备成了自然而然的事情。虽然大多数汇编流程不需要为了在此类设备上实现概念验证而更改，但是运行时间不能依赖于：</p>
<ul class="simple">
<li><p><strong>虚拟内存</strong>，和其他任何系统提供的 <code class="docutils literal notranslate"><span class="pre">malloc</span></code>。另外，微控制器设备的内存资源通常非常有限（以 KB 为单位）。因此，为此类平台设计的库在使用内存时通常需要非常谨慎，并且需要在内存未使用时及时释放内存。</p></li>
<li><p>传统的操作系统功能，如 <strong>files</strong>，<strong>libraries</strong> 和 <strong>kernel functions</strong>。一些项目实现了对这些的支持，但它们绝不是标准的。</p></li>
<li><p>支持除 <strong>C</strong> 以外的编程语言。</p></li>
</ul>
<p>这些变化需要一个不同于传统操作系统通常使用的 TVM C++运行时间。</p>
</div>
<div class="section" id="typical-use">
<h2><a class="toc-backref" href="#id3">典型案例</a><a class="headerlink" href="#typical-use" title="永久链接至标题">¶</a></h2>
<p>This section discusses our vision of the “typical” microTVM use case. Each component used to achieve
this typical use case is intended to be designed for flexibility, but this unifying vision serves to
motivate the inclusion of each part of the design.</p>
<div class="figure align-center">
<a class="reference internal image-reference" href="https://raw.githubusercontent.com/tvmai/web-data/main/images/dev/microtvm_workflow.svg"><img alt="https://raw.githubusercontent.com/tvmai/web-data/main/images/dev/microtvm_workflow.svg" src="https://raw.githubusercontent.com/tvmai/web-data/main/images/dev/microtvm_workflow.svg" width="85%" /></a>
</div>
<p>The parts of this process are described below:</p>
<ol class="arabic simple">
<li><p><strong>Model Import</strong>. The user imports an existing model or describes a new model to TVM, producing a
<em>Relay module</em>.</p></li>
<li><p><strong>Model Transformations</strong>. The user can apply transformations, such as quantization, to the
model. After each transformation, the user should still have a Relay module.</p></li>
<li><p><strong>Compilation</strong> (Scheduling and Code Generation). TVM implements each operator into Tensor IR by
assigning a schedule and schedule configuration to each Relay operator. Then, code (C source or
compiled object) is generated for each operator.</p></li>
<li><p><strong>Integration</strong>. The generated code is integrated along with the TVM C Runtime library into a
user-supplied binary project. In some cases (such as when the project is standardized across
multiple SoC/development boards), this process is handled automatically.</p></li>
<li><p><strong>Deployment</strong>. The project is built and the residual firmware binary is flashed onto the device.
Model inference is driven either by TVM using an on-device RPC server, or on the device using the
on-device Graph Executor.</p></li>
</ol>
</div>
<div class="section" id="design-goals">
<h2><a class="toc-backref" href="#id4">Design Goals</a><a class="headerlink" href="#design-goals" title="永久链接至标题">¶</a></h2>
<p>microTVM aims to achieve these design goals:</p>
<ol class="arabic simple">
<li><p><strong>Portable Code</strong>. microTVM can translate any Relay model into C code that can compile with only
a C standard library.</p></li>
<li><p><strong>Minimal Overhead</strong>. microTVM generates target-specific, highly optimized code. As much overhead
from the runtime should be removed.</p></li>
<li><p><strong>Accessible Code</strong>. microTVM considers C source code as a first-class output mechanism so that
it is easier for a firmware engineer to understand and tweak.</p></li>
</ol>
</div>
<div class="section" id="overview">
<h2><a class="toc-backref" href="#id5">Overview</a><a class="headerlink" href="#overview" title="永久链接至标题">¶</a></h2>
<p>microTVM requires changes at all levels of the TVM compiler stack. The following sub-sections enumerate
these changes at a high level, and follow-on sections discuss the specifics in more detail.</p>
<div class="section" id="modeling-target-platforms">
<h3><a class="toc-backref" href="#id6">Modeling Target Platforms</a><a class="headerlink" href="#modeling-target-platforms" title="永久链接至标题">¶</a></h3>
<p>TVM’s search-based optimization approach allows it to largely avoid system-level modeling of targets
in favor of experimental results. However, some modeling is necessary in order to ensure TVM is
comparing apples-to-apples search results, and to avoid wasting time during the search by attempting
to compile invalid code for a target.</p>
<p>microTVM models these parts of the target:</p>
<ul class="simple">
<li><p>The CPU used, through the <code class="docutils literal notranslate"><span class="pre">-mcpu</span></code> and <code class="docutils literal notranslate"><span class="pre">-march</span></code> target flags.</p></li>
<li><p>The presence or absence of accelerators, through the device components of the target (Currently
only the absence of accelerators can be expressed, but this mechanism should extend well).</p></li>
</ul>
<p>microTVM aims to model these parts of the target in the future:</p>
<ul class="simple">
<li><p>Memory, modeled as a set of disjoint memory spaces, each with a label and size and prefetch/flush
behavior. Some memory may be shared with accelerators.</p></li>
<li><p>Target runtime configuration (i.e. clock tree configuration, clock speed, etc). This is intended
only to contribute to the AutoTVM schedule key and not for any other use.</p></li>
</ul>
<p>At this time, TVM does not intend to model:</p>
<ul class="simple">
<li><p>Size, type, or relationship of caches, with the exception of prefetching or cache flushing.</p></li>
</ul>
</div>
<div class="section" id="tvm-targets-for-microtvm">
<h3><a class="toc-backref" href="#id7">TVM Targets for microTVM</a><a class="headerlink" href="#tvm-targets-for-microtvm" title="永久链接至标题">¶</a></h3>
<p>A central data structure in the compilation process is the <code class="docutils literal notranslate"><span class="pre">tvm::target::Target</span></code> class. TVM uses
Target to decide which TIR schedules to enable and how to configure the code generator. The Target
class should also uniquely identify the generated code for a particular operator, as autotuning
logs use it to rank measured performance (but see Future Work).</p>
<p>Targets are currently represented as strings structured similarly to command-line arguments. An
example target is shown below:</p>
<blockquote>
<div><p><code class="docutils literal notranslate"><span class="pre">c</span> <span class="pre">-keys=arm_cpu</span> <span class="pre">-mcpu=cortex-m7</span> <span class="pre">-link-params</span> <span class="pre">-model=stm32f746xx</span> <span class="pre">-runtime=c</span> <span class="pre">-system-lib=1</span></code></p>
</div></blockquote>
<p>The relevant parts to microTVM are:</p>
<blockquote>
<div><ul class="simple">
<li><p>Code generator (<code class="docutils literal notranslate"><span class="pre">llvm</span></code> or <code class="docutils literal notranslate"><span class="pre">c</span></code>)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">-mcpu=cortex-m7</span></code>: used by TOPI to enable Cortex-M schedules, and, when the C source code
generator is selected, included in the output as a comment to help identify the code and
configure the downstream C compiler.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">-link-params</span></code>: include parameters as global constants to load from flash.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">-runtime=c</span></code>: build glue code to allow operators to work with the C runtime</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">-system-lib=1</span></code>: emit a system library (i.e. which can be loaded by calling the PackedFunc
<code class="docutils literal notranslate"><span class="pre">runtime.SystemLib</span></code>.</p></li>
</ul>
</div></blockquote>
</div>
<div class="section" id="writing-schedules-for-microtvm">
<h3><a class="toc-backref" href="#id8">Writing Schedules for microTVM</a><a class="headerlink" href="#writing-schedules-for-microtvm" title="永久链接至标题">¶</a></h3>
<p>For operations scheduled on the CPU, microTVM initially plans to make use of specialized
instructions and extern (i.e. hand-optimized) functions to achieve good performance. In TVM, this
approach is generally accomplished through tensorization, in which TVM breaks a computation into
small pieces, and a TIR extern function accelerates each small piece.</p>
<p>TVM currently accommodates both approaches using <code class="docutils literal notranslate"><span class="pre">tir.call_extern</span></code>. First, a pragma is attached to
the schedule defining the extern function in portable C.</p>
<blockquote>
<div><p><code class="docutils literal notranslate"><span class="pre">sched[output].pragma(n,</span> <span class="pre">&quot;import_c&quot;,</span> <span class="pre">&quot;void</span> <span class="pre">call_asm(int32_t*</span> <span class="pre">a,</span> <span class="pre">int32_t*</span> <span class="pre">b)</span> <span class="pre">{</span> <span class="pre">/*</span> <span class="pre">...</span> <span class="pre">*/</span> <span class="pre">}&quot;)</span></code></p>
</div></blockquote>
<p>Next, <code class="docutils literal notranslate"><span class="pre">tensorize</span></code> is used to split the computation.</p>
<blockquote>
<div><p><code class="docutils literal notranslate"><span class="pre">sched[output].tensorize(owi,</span> <span class="pre">gemm)</span></code></p>
</div></blockquote>
<p>There are a couple of caveats to this approach, all which could be resolved by linking generated
code against external libraries:</p>
<ul class="simple">
<li><p>Inline assembly is compiler-specific. While Clang and GCC have standardized on one syntax, this
may not be portable to other compilers. SDKs solve this by conditionally including a header file
depending on the compiler being used. However, taking this approach means that the generated code
needs additional compiler flags (i.e. <code class="docutils literal notranslate"><span class="pre">-Isystempath/to/header</span></code>).</p></li>
<li><p>It may be helpful to reference helper functions from the generated code (e.g. to inline common
sequences of hand-optimized assembly).</p></li>
<li><p>Finally, the extern function invoked may be wholly written in an external library. If those
functions can be wholly inlined, this caveat is the same as the previous. If not, then additional
C code needs to be compiled and linked against the operator.</p></li>
</ul>
<p>At present, microTVM presumes that all eligible schedules can be compiled. This means that the user-
supplied project (see next section) must include all libraries that are used by the generated code.
When not using autotuning, TVM randomly chooses a fallback schedule, so all libraries would need to
be supported. When using autotuning, TVM selects the best-performing schedule, so only that library
is needed. There isn’t currently a way to force TVM to pick a particular schedule outside of
autotuning logs, but that would be a good addition.</p>
<p>Finally, when using the <code class="docutils literal notranslate"><span class="pre">llvm</span></code> backend, the process is similar except that LLVM bitcode is included
in the generated code (with an <code class="docutils literal notranslate"><span class="pre">import_llvm</span></code> pragma). LLVM bitcode provides a portable way to call
inline assembly. However, it may be more complex to call external C functions, and helper functions
are of course not easy to use from LLVM bitcode.</p>
</div>
<div class="section" id="executing-models">
<h3><a class="toc-backref" href="#id9">Executing Models</a><a class="headerlink" href="#executing-models" title="永久链接至标题">¶</a></h3>
<p>The TVM compiler traditionally outputs three pieces:</p>
<ol class="arabic simple">
<li><p>Model operator implementations, as discussed above;</p></li>
<li><p>A model execution graph, encoded as JSON; and</p></li>
<li><p>Simplified parameters.</p></li>
</ol>
<p>To correctly execute the model, a Graph Executor needs to reconstruct the graph in memory, load the
parameters, and then invoke the operator implementations in the correct order.</p>
<p>microTVM supports two ways to do this:</p>
<ol class="arabic simple">
<li><p><strong>Host-Driven</strong>. The Graph Executor can run on the host and carry out execution by issuing
commands to the device using an RPC link with a UART-like transport.</p></li>
<li><p><strong>Standalone</strong>. A C Graph Executor is available to be compiled on-device, but it is not
particularly memory efficient. This way enables standalone execution without any attached host.</p></li>
</ol>
<p>Host-Driven is designed for experimenting with models on-device and, like AutoTVM, uses the RPC server to
drive computation on-device. Standalone is intended for deployment.</p>
<div class="section" id="host-driven-execution">
<h4>Host-Driven Execution<a class="headerlink" href="#host-driven-execution" title="永久链接至标题">¶</a></h4>
<p>In Host-Driven execution, the firmware binary is the following:</p>
<ol class="arabic simple">
<li><p>Generated operator implementations from TVM.</p></li>
<li><p>The TVM C runtime.</p></li>
<li><p>SoC-specific initialization.</p></li>
<li><p>The TVM RPC server.</p></li>
<li><p>(optional) Simplified Parameters.</p></li>
</ol>
<p>This firmware image is flashed onto the device and a GraphExecutor instance is created on the host.
The GraphExecutor drives execution by sending RPC commands over a UART:</p>
<div class="figure align-center">
<a class="reference internal image-reference" href="https://raw.githubusercontent.com/tvmai/web-data/main/images/dev/microtvm_host_driven.svg"><img alt="https://raw.githubusercontent.com/tvmai/web-data/main/images/dev/microtvm_host_driven.svg" src="https://raw.githubusercontent.com/tvmai/web-data/main/images/dev/microtvm_host_driven.svg" width="85%" /></a>
</div>
</div>
<div class="section" id="standalone-execution">
<h4>Standalone Execution<a class="headerlink" href="#standalone-execution" title="永久链接至标题">¶</a></h4>
<p>In Standalone execution, the GraphExecutor is instantiated on device:</p>
<div class="figure align-center">
<a class="reference internal image-reference" href="https://raw.githubusercontent.com/tvmai/web-data/main/images/dev/microtvm_standalone.svg"><img alt="https://raw.githubusercontent.com/tvmai/web-data/main/images/dev/microtvm_standalone.svg" src="https://raw.githubusercontent.com/tvmai/web-data/main/images/dev/microtvm_standalone.svg" width="85%" /></a>
</div>
</div>
</div>
<div class="section" id="microtvm-firmware">
<h3><a class="toc-backref" href="#id10">microTVM Firmware</a><a class="headerlink" href="#microtvm-firmware" title="永久链接至标题">¶</a></h3>
<p>We can now discuss how microTVM firmware should behave. An important task common to both model
execution strategies is configuring the SoC to match the way it performs in production. microTVM
considers this task project- and SoC-dependent. Whether for AutoTVM, host-driven model inference, or
in standalone deployment, the user is expected to supply a project whose main() does the following:</p>
<ol class="arabic simple">
<li><p>Configure the SoC to match deployment performance.</p></li>
<li><p>Initialize the TVM C Runtime.</p></li>
</ol>
<p>When configuring for host-driven inference or AutoTVM, the remaining tasks are well-defined:</p>
<ol class="arabic simple" start="3">
<li><p>Initialize a transport (i.e. a UART) for use with the TVM RPC server.</p></li>
<li><p>Launch the TVM RPC Server.</p></li>
</ol>
<p>When configuring for standalone deployment, the firmware needs to:</p>
<ol class="arabic simple">
<li><p>Instantiate the system library by calling the <code class="docutils literal notranslate"><span class="pre">runtime.SystemLib</span></code> PackedFunc.</p></li>
<li><p>Instantiate a GraphExecutor passing the system library module.</p></li>
<li><p>Configure parameters and inputs as needed.</p></li>
<li><p>Run the model.</p></li>
</ol>
</div>
<div class="section" id="parts-of-a-microtvm-binary">
<h3><a class="toc-backref" href="#id11">Parts of a microTVM Binary</a><a class="headerlink" href="#parts-of-a-microtvm-binary" title="永久链接至标题">¶</a></h3>
<p>To summarize, a microTVM firwmare binary image must contain these parts:</p>
<ol class="arabic simple">
<li><p>Operator implementations, produced by TVM.</p></li>
<li><p>The TVM C runtime library, supplied by TVM as a static library.</p></li>
<li><p>SoC Initialization, supplied by the user.</p></li>
</ol>
<p>For Host-driven model execution, firmware also needs:</p>
<ol class="arabic simple" start="4">
<li><p>The TVM RPC Server library.</p></li>
</ol>
<p>For Standalone model execution, firmware also needs:</p>
<ol class="arabic simple" start="4">
<li><p>The TVM C GraphExecutor library, supplied by TVM as a static library.</p></li>
<li><p>The remaining compiler outputs (Simplified Parameters and Graph JSON).</p></li>
</ol>
</div>
<div class="section" id="the-automated-build-flow">
<h3><a class="toc-backref" href="#id12">The Automated Build Flow</a><a class="headerlink" href="#the-automated-build-flow" title="永久链接至标题">¶</a></h3>
<p>Once code generation is complete, <code class="docutils literal notranslate"><span class="pre">tvm.relay.build</span></code> returns a <code class="docutils literal notranslate"><span class="pre">tvm.runtime.Module</span></code> and the
user can save the generated C source or binary object to a <code class="docutils literal notranslate"><span class="pre">.c</span></code> or <code class="docutils literal notranslate"><span class="pre">.o</span></code> file. From this point, TVM
can theoretically step back and the user can compile and run the code separately.</p>
<p>However, for AutoTVM, TVM needs some automated flow to handle the following tasks:</p>
<ol class="arabic simple">
<li><p>Integrate operator implementations, the TVM C Runtime library, and the TVM RPC Server library into the
firmware project containing user-supplied SoC Initialization.</p></li>
<li><p>Build the resulting project.</p></li>
<li><p>Program the built firmware onto a (specific) attached device.</p></li>
<li><p>Identify the serial port or other transport to be used by TVM to drive remote execution.</p></li>
</ol>
<p>At present, TVM expects the user to supply an implementation of the <code class="docutils literal notranslate"><span class="pre">tvm.micro.Compiler</span></code>,
<code class="docutils literal notranslate"><span class="pre">tvm.micro.Flasher</span></code>, and <code class="docutils literal notranslate"><span class="pre">tvm.micro.Transport</span></code> interfaces. TVM then:</p>
<ol class="arabic simple">
<li><p>Builds each piece separately as a library.</p></li>
<li><p>Builds the libraries into a binary firmware image.</p></li>
<li><p>Programs the firmware image onto an attached device.</p></li>
<li><p>Opens a serial port to serve as the RPC server transport.</p></li>
</ol>
<p>This design was chosen to reduce build times for microTVM (the common libraries need to be built
only once per candidate operator implemmentation). In practice, these projects are extremely small
and compile relatively quickly. Compared with the added complexity of this tighter build integration
with TVM, the performance gains are likely not worth it. A future design will consolidate the build
tasks into a single step and narrow the interface to provide a better integration.</p>
</div>
<div class="section" id="measuring-operator-performance">
<h3><a class="toc-backref" href="#id13">Measuring operator performance</a><a class="headerlink" href="#measuring-operator-performance" title="永久链接至标题">¶</a></h3>
<p>The TVM C runtime depends on user-supplied functions to measure time on-device. Users should implement
<code class="docutils literal notranslate"><span class="pre">TVMPlatformTimerStart</span></code> and <code class="docutils literal notranslate"><span class="pre">TVMPlatformTimerStop</span></code>. These functions should measure wall clock time, so there
are some pitfalls in implementing these functions:</p>
<ol class="arabic simple">
<li><p>If the CPU could halt or sleep during a computation (i.e. if it is being done on an accelerator),
a cycle counter should likely not be used as these tend to stop counting while the CPU is asleep.</p></li>
<li><p>The granularity of these functions can be relaxed as needed to extend the range of the timer
device. However, if granularity is too coarse, a sub-optimal schedule may be used.</p></li>
<li><p>An error should be raised if the timer overflows.</p></li>
<li><p>The timer should not interrupt computation unless absolutely necessary. Doing so may affect the
accuracy of the results.</p></li>
<li><p>Calibrating the output against a wall clock is ideal, but it will likely be too cumbersome. A
future PR could enable some characterization of the platform timer by, e.g., measuring the internal
oscillator against a reference such as an external crystal.</p></li>
</ol>
</div>
</div>
<div class="section" id="future-work">
<h2><a class="toc-backref" href="#id14">Future Work</a><a class="headerlink" href="#future-work" title="永久链接至标题">¶</a></h2>
<div class="section" id="ahead-of-time-runtime">
<h3><a class="toc-backref" href="#id15">Ahead-of-Time Runtime</a><a class="headerlink" href="#ahead-of-time-runtime" title="永久链接至标题">¶</a></h3>
<p>A limitation of the Graph Executor is the amount of memory overhead required in parsing the JSON.
The current implementation contributes significantly to the dynamic memory usage of microTVM,
limiting its utility. An ahead-of-time runtime can avoid the need for any Graph JSON parsing and
improve inference speed by generating C code to call the generated operator implementations directly
rather than relying on a data-driven approach with the Graph Executor.</p>
</div>
<div class="section" id="memory-planning">
<h3><a class="toc-backref" href="#id16">Memory Planning</a><a class="headerlink" href="#memory-planning" title="永久链接至标题">¶</a></h3>
<p>The current memory planner attempts to limit the number of <code class="docutils literal notranslate"><span class="pre">TVMBackendDeviceAlloc()</span></code> calls
issued for intermediate tensors only. Because scratchpads can vary widely, and because the planner
coalesces memory allocations within 16x of each other, this strategy typically results in high
peak memory usage.</p>
</div>
<div class="section" id="heterogeneous-execution">
<h3><a class="toc-backref" href="#id17">Heterogeneous Execution</a><a class="headerlink" href="#heterogeneous-execution" title="永久链接至标题">¶</a></h3>
<p>Newer Cortex-M SoCs can contain multiple CPUs and onboard ML accelerators.</p>
</div>
<div class="section" id="autotuning-target">
<h3><a class="toc-backref" href="#id18">Autotuning Target</a><a class="headerlink" href="#autotuning-target" title="永久链接至标题">¶</a></h3>
<p>As discussed previously,</p>
</div>
</div>
</div>


           </div>
           
          </div>
          

<footer>

    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="model_library_format.html" class="btn btn-neutral float-right" title="Model Library Format" accesskey="n" rel="next">下一个 <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="security.html" class="btn btn-neutral float-left" title="Security Guide" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> 上一个</a>
      
    </div>

<div id="button" class="backtop"><img src="../_static//img/right.svg" alt="backtop"/> </div>
<section class="footerSec">
    <div class="footerHeader">
      <ul class="d-flex align-md-items-center justify-content-between flex-column flex-md-row">
        <li class="copywrite d-flex align-items-center">
          <h5 id="copy-right-info">© 2020 Apache Software Foundation | All right reserved</h5>
        </li>
      </ul>

    </div>

    <ul>
      <li class="footernote">Copyright © 2020 The Apache Software Foundation. Apache TVM, Apache, the Apache feather, and the Apache TVM project logo are either trademarks or registered trademarks of the Apache Software Foundation.</li>
    </ul>

</section>
</footer>
        </div>
      </div>

    </section>

  </div>
  

    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>

  </body>
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
    <!-- Theme Analytics -->
    <script>
    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
    })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

    ga('create', 'UA-75982049-2', 'auto');
    ga('send', 'pageview');
    </script>

    
   

</body>
</html>